package com.foo.preprocessing;

import java.io.File;
import java.util.regex.Pattern;

import com.foo.constants.Constants;

import weka.core.Instances;
import weka.core.pmml.Constant;
import weka.core.stemmers.LovinsStemmer;
import weka.filters.Filter;
import weka.filters.unsupervised.attribute.RemoveByName;
import weka.filters.unsupervised.attribute.Reorder;
import weka.filters.unsupervised.attribute.ReplaceMissingValues;
import weka.filters.unsupervised.attribute.StringToNominal;
import weka.filters.unsupervised.attribute.StringToWordVector;
import weka.gui.SelectedTagEditor;

public class PreprocessedDataset 
{
	
	/*
	 * Applies StringToNominal Filter on the instance entered
	 * It Applies StringToNominal on the last attribute i.e category
	 * Input : Un-filtered Instance Data
	 * Output: Filtered Instance Data
	 */
	public Instances Apply_String_To_Nominal_Filters(Instances unFilteredData) throws Exception
	{
		Instances filteredData = null;
		try
		{
			StringToNominal filter = new StringToNominal();
			filter.setAttributeRange("last");
			filter.setInputFormat(unFilteredData);
			filteredData = Filter.useFilter(unFilteredData, filter);
		}
		catch(Exception e)
		{
			System.err.println("Error when applying StringToNominal Filter: " + e.getMessage());
		}
		return filteredData;
	}

	/*
	 * Applies StringToWordVector filter on the instance entered
	 * It ensures that in the string attribute are pre-processed by removing stop words and
	 * tokenizing the attribute value so that each word will be considered as an attribute.
	 * It also removes all the attributes starting from numbers
	 */
	public Instances Apply_String_To_Vector_Filters(Instances unFilteredData) throws Exception
	{
		Instances filteredData = null;
		try
		{			
			StringToWordVector filter = new StringToWordVector();
			filter.setInputFormat(unFilteredData);
			
			filter.setMinTermFreq(Constants.Min_Term_Frequency);
			filter.setIDFTransform(Constants.IDF_Transform);
			filter.setLowerCaseTokens(Constants.Lower_Case_Tokens);
			filter.setTFTransform(Constants.TF_Transform);
			filter.setUseStoplist(Constants.USE_STOP_LIST);
			int lastindex = unFilteredData.numAttributes() -1;
			filter.setAttributeIndices("first-"+ lastindex);
			LovinsStemmer stemmer = new LovinsStemmer();
			filter.setStemmer(stemmer);
			
			filteredData = Filter.useFilter(unFilteredData, filter);
			
			filteredData.setClass(filteredData.attribute(Constants.CATEGORY_ATTRIBUTE));
			
			// Removing unwanted attributes attributes starting from numbers, special characters etc
			RemoveByName remove_attributes = new RemoveByName();
			remove_attributes.setInputFormat(filteredData);
			remove_attributes.setExpression(Constants.REGULAR_EXPRESSION);
			remove_attributes.setInvertSelection(Constants.INVERT_SELECTION);
			
			filteredData = Filter.useFilter(filteredData, remove_attributes);
			
			//Reorder the class attribute location. This sets the category attribute to the last location
			Reorder reorder_attributes = new Reorder();
			reorder_attributes.setAttributeIndices("last-first");
			reorder_attributes.setInputFormat(filteredData);
			filteredData = Filter.useFilter(filteredData, reorder_attributes);
			
			//Replace Missing values
			ReplaceMissingValues replace_Missing = new ReplaceMissingValues();
			replace_Missing.setInputFormat(filteredData);
			
			filteredData = Filter.useFilter(filteredData, replace_Missing);
		}
		catch(Exception e)
		{
			System.err.println("Error when applying StringToNomial Filter : "+ e.getMessage());
		}
		return filteredData;
	}
	
	
	
}
